r"""Module to allow a user on a unix system to interrupt a running
process, connect to it, inspect variables, and then restart.
Based on a recipe by Brian McErlean:
http://code.activestate.com/recipes/576515/
Usage
-----
Start a process and run :func:`listen`. This will register a signal
handler and print out a message about what signal to send:
>>> import mmf.async.remote_debug
>>> mmf.async.remote_debug.example() # doctest: +SKIP
Debug listener started for process 12578.
Start winpdb and send signal 30 to process 12578.
With a shell you can do this as:
winpdb& kill -%i %i
Original Documentation
----------------------
This provides code to allow any python program which uses it to be
interrupted at the current point, and communicated with via a normal
python interactive console. This allows the locals, globals and
associated program state to be investigated, as well as calling
arbitrary functions and classes.
To use, a process should import the module, and call listen() at any
point during startup. To interrupt this process, the script can be run
directly, giving the process Id of the process to debug as the
parameter.
Discussion
~~~~~~~~~~
This was written to deal with a problem I had with a long-running
background process that would occassionally get stuck after a long
time. This was difficult to reproduce when testing, so I wrote the
above code so that when it did happen, I'd be able to break in and see
what was going on.
Its implemented by first sending the process to debug a signal, and
then opening a pair of pipes with the name `/tmp/debug-pid.in` and
`/tmp/debug-pid.out`. The remote process, on receiving the signal, opens
the other end of this pipe and these are used to pass code to be
executed from the debugging process, and read responses from the
debugee.
There are a few warnings to make:
* There is absolutely no security here - pretty much anyone who can
write to the pipe can gain full control of any process using
this. Use only for developer environments, not live systems!
* Sending a signal can interrupt whatever I/O or activity the process
is currently doing, so you won't always just be able to detach again
and let it run unchanged.
* It uses signals to wake the process, so currently only works on
unix-like systems that support this.
* Untested with threads
Updated Documentation
---------------------
Details of Operation
~~~~~~~~~~~~~~~~~~~~
The way this works is that the original process calls :func:`listen`
which registers a signal handler -- either :func:`_remote_pdb_handler`
or :func:`_pdb_handler` -- to be called when it receives
:attr:`_REMOTE_SIGNAL`.
The handler then creates an instance of :class:`pdb.Pdb` attached to
two named handles `name'.in'` and `name'.out'` where `name` is
returned by :func:`_pipename` and contains the process id: Thus there
is a unique pair of named pipes for each process that runs
:func:`listen`.
Another process can then be used to remotely debug the process by:
1) sending signal :attr:`_REMOTE_SIGNAL` to the process.
2) reading output from the named pipe `name.out`
3) sending pdb commands to the named pipe `name.in`
This could all be done using the shell as follows::
$ kill -_REMOTE_SIGNAL pid
$ cat < name.out& cat > name.in
There are some issues with the persistence of the pipes etc.
Originally we created these in the handler, but then the execution
halts in the handler and as soon as the handler exits, the pipes are
closed. We now rely on Pdb to close the pipes when it exits, and rely
on an :mod:`atexit` hook to delete the pipes (which means that they
may still remain if the original process is killed suddenly).
Examples
--------
Known Bugs
----------
.. todo:: Make the handlers check to see if the process is already
being debugged. If so, don't start a new debugger (otherwise one
will start debugging the debugger internals!) Print a message for
the user.
.. todo:: Make a pythonic version of the client that will ensure the
streams are properly closed. Right now, Ctrl-c on the terminal
will hang the process.
"""
import signal
import sys
import cPickle
import cStringIO
import codeop
import os
import tempfile
import time
import traceback
import pdb, bdb
try:
import readline # For readline input support if available
except ImportError:
pass
def import_rpdb2():
r"""This is not a "safe" import. It has side-effects such as installing its
own importer. Thus, we only do when actually listening (as opposed to
documenting this module.)"""
rpdb2 = None
try:
import rpdb2
except ImportError:
pass
return rpdb2
__all__ = ['listen']
_REMOTE_SIGNAL = signal.SIGUSR1 # Signal used to start remote debugging
_PDB_SIGNAL = signal.SIGUSR2 # Signal used to start pdb on process
# (uses the processes original stdin
# and stdout)
_RPDB2_SIGNAL = signal.SIGUSR1 # Signal to start rpdb2 debugging
_DEBUG = True
_PASSWORD = "password" # Password for rpdb2 remote debugging
# None will prompt.
_RPDB2_TIMEOUT = 60 # Timeout for connections to rpdb2
_RPDB2_ALLOW_REMOTE = True # Default for allowing remote debugging
def _pipename(pid):
"""Return name of pipe to use"""
return os.path.join(tempfile.gettempdir(), 'debug-%d' % pid)
class Pdb(pdb.Pdb):
r"""Subclass of :class:`pdb.Pdb` that is told not to throw
exceptions (hence not accidentally killing the parent process!)"""
def trace_dispatch(self, *v, **kw):
print "Here"
try:
pdb.Pdb.trace_dispatch(self, *v, **kw)
return self.trace_dispatch
except:
# Ignore exceptions
if self.quitting:
while _NamedPipe._NAMED_PIPES:
_NamedPipe._NAMED_PIPES.pop().close()
return None
def set_continue(self):
# Don't stop except at breakpoints or when finished
self.stopframe = self.botframe
self.returnframe = None
self.quitting = 0
if not self.breaks:
# no breakpoints; quit.
self.set_quit()
def get_named_pipe(name, end=0, mode=0666):
"""Return a named pipe.
This checks to see if the specified pipe exists, otherwise, it
return a new named pipe object."""
if _NamedPipe._NAMED_PIPES:
# Pipe might already exist
for pipe in _NamedPipe._NAMED_PIPES:
if (pipe.name == name and pipe.end == end and
pipe.is_open()):
if _DEBUG: print("* Returning existing _NamedPipe")
if mode != pipe.mode:
warnings.warn("_NamedPipe exists but with "
"different mode=%i than requested (%i)" %
(pipe.mode, mode))
return pipe
return _NamedPipe(name, end, mode)
class _NamedPipe(object):
_NAMED_PIPES = []
def __init__(self, name, end=0, mode=0666):
"""Open a pair of pipes, `name.in` and `name.out` for communication
with another process. One process should pass 1 for end, and the
other 0. Data is marshalled with pickle.
The pipes are closed and the files are deleted upon deletion of
this object, but this needs to be a late operation (after all
debugging activities are finished), so the object should be
held in a global variable, to be deleted when the process
terminates or the module is removed.
"""
self.name = name
self.end = end
self.mode = mode
if _DEBUG: print("* _NamedPipe(%s, %i) called " % (name, end))
self.in_name, self.out_name = name +'.in', name +'.out',
try:
os.mkfifo(self.in_name, mode)
except OSError:
pass
try:
os.mkfifo(self.out_name, mode)
except OSError:
pass
# NOTE: The order the ends are opened in is important - both ends
# of pipe 1 must be opened before the second pipe can be opened.
if end:
self.inp = open(self.out_name,'r')
self.out = open(self.in_name,'w')
else:
self.out = open(self.out_name,'w')
self.inp = open(self.in_name,'r')
_NamedPipe._NAMED_PIPES.append(self)
def is_open(self):
return not (self.inp.closed or self.out.closed)
def close(self):
if _DEBUG: print("* Closing _NamedPipe(%s,%i)" %
(self.name, self.end))
self.inp.close()
self.out.close()
try:
os.remove(self.in_name)
except OSError:
pass
try:
os.remove(self.out_name)
except OSError:
pass
def __del__(self):
if _DEBUG: print("* Deleting _NamedPipe(%s,%i)" %
(self.name, self.end))
self.close()
def debug_process(pid):
"""Interrupt a running process and debug it."""
pipename = _pipename(pid)
print("Named pipes opened in %s.[in|out]" % pipename)
print("Use as follows:")
print("kill -%i %i;cat < %s.out &cat > %s.in;fg" %
((_REMOTE_SIGNAL, pid) + (pipename, )*2))
#os.kill(pid, _SIGNAL) # Signal process to start debugger.
def get_rpdb2_handler(password=None,
allow_remote=_RPDB2_ALLOW_REMOTE,
timeout=_RPDB2_TIMEOUT):
r"""Return a handler that starts an embedded rpdb2 debugger
listening with the specified password."""
if password is None:
if sys.stdout is not None:
sys.stdout.write('Please type password:')
password = sys.stdin.readline().rstrip('\n')
def _rpdb2_handler(sig, frame,
_password=password,
_allow_remote=allow_remote,
_timeout=timeout):
r"""If :mod:`rpdb2` is available (or :mod:`winpdb`) then this
handler can be used to provide a much more secure and functional
debugging environment."""
rpdb2 = import_rpdb2()
rpdb2.start_embedded_debugger(_password,
fAllowUnencrypted=True,
fAllowRemote=_allow_remote,
timeout=_timeout,
fDebug=False)
return _rpdb2_handler
def winpdb(pid):
r"""Start the winpdb GUI and trigger the specified process to
start listening."""
import winpdb
os.kill(pid, _RPDB2_SIGNAL) # Signal process to start debugger.
print("Go to File/Attach to see the process.")
sys.argv = ['winpdb']
winpdb.main()
def _remote_pdb_handler(sig, frame):
pid = os.getpid() # Use pipe name based on pid
name = _pipename(pid)
pipe = get_named_pipe(name)
print("\n".join([ "Debug listener started for process %i." % (pid,),
"Debug with the following commands:",
""
"kill -%i %i" % (_REMOTE_SIGNAL, pid),
"cat < %s.out&" % (name,),
"cat > %s.in" % (name,)]))
Pdb(stdin=pipe.inp, stdout=pipe.out).set_trace(frame)
def _pdb_handler(sig, frame):
"""This starts pdb on the remote process."""
Pdb().set_trace(frame)
[docs]def listen(password=_PASSWORD, verbose=True):
r"""Start debug listener for the current process.
This registers a signal handler so that :func:`debug_process` can
be used to inspect the current process.
Parameters
----------
password : str, optional
This is the password required to connect with :mod:`winpdb`.
If it is not provided, then the user will be prompted for one
right now on the running process.
verbose: bool, optional
If `True` (default), then a message with the current process id
will be displayed.
"""
pid = os.getpid()
rpdb2 = import_rpdb2()
if rpdb2 is not None:
signal.signal(_RPDB2_SIGNAL, get_rpdb2_handler(password))
mesg = "\n".join([
"Start winpdb and send signal %i to process %i." % (
_RPDB2_SIGNAL, pid),
"With a shell you can do this as:",
"",
"winpdb&kill -%i %i" % (_RPDB2_SIGNAL, pid)])
else:
signal.signal(_REMOTE_SIGNAL, _remote_pdb_handler)
mesg = "\n".join([
"With a shell you can debug as follows:",
"",
"kill -%i %i" % (_REMOTE_SIGNAL, pid),
"cat < %s.out&" % (_pipename(pid),),
"cat > %s.in" % (_pipename(pid),)])
# Register for remote debugging.
signal.signal(_PDB_SIGNAL, _pdb_handler)
if verbose:
print("\n".join([ "Debug listener started for process %i."
% (pid,),
mesg]))
if __name__=='__main__':
if len(sys.argv) != 2:
print "Error: Must provide process id to debug"
else:
pid_ = int(sys.argv[1])
debug_process(pid_)
def example():
r"""Simply counts and prints the results to the screen. This can
be interrupted and debugged by a separate process."""
import mmf.utils
sb = mmf.utils.StatusBar().new_bar()
i = 0
# Start the process listening. This will print a message about
# how to listen.
listen()
halt = [False]
while 1:
sb.msg(str(i))
i += 1
if halt[0]:
break